const cheerio = require("cheerio");
const nssd = require("../lib/nssd");
const http = require("../lib/httpClien");
const httpClien = new http();
const BasicUrl = "http://www.nssd.org/";

//页面主页（待建）
exports.index = async function  (ctx, next) {
    await ctx.render('index', {
        session: ctx.session,
    })
    // ctx.body = "here is index!";
}

exports.crawler = async function  (ctx, next) {

    let {pageS = 1 ,pageE = 0, keyword = "马克思主义"} = ctx.request.body;
    await nssdCrawler(pageS,pageE,keyword);

    return ctx.body = {
        code : 0,
        message : "success"
    };
}

const nssdCrawler = async function (pageS,pageE,keyword) {
    console.log('start');
    let p = pageS
    for (; p <= pageE; p++) { //页码
        let url = `${BasicUrl}articles/articlesearch.aspx?invokemethod=search&q={"search":"${keyword}","sType":"all","page":"${p}"}&&hidpage=0&&hfldSelectedIds=&`;  //关键词
        console.log(url)
        try {
            await _do(encodeURI(url));
        } catch (err) {
            console.log(err);
        }
    }

    console.log('end');
    // ctx.body = "end！";
}

const _do = async function (url, ctx=null) {
    let list = await httpClien.get(url);
    if (!list) {
        ctx && ctx.websocket.send(`${url}:list empty!`);
        console.log(`${url}:list empty!`);
        return false;
    }

    ctx && ctx.websocket.send(`${url}:已载入服务器...`);
    console.log("load");
    var $ = await cheerio.load(list.data);
    //通过CSS selector来筛选数据
    await $('a.title').each(async function (idx, element) {
        await sleep(300);
        let subLink = element.attribs.href;
        let detailLink = BasicUrl + subLink;
        let id = subLink.match(/\d+/);

        if (!id) {
            return true;
        }
        let record = await nssd.findNssdData(id[0]);
        if (record[0] !== undefined) {
            return true;
        }
        //获取目录
        let catLink = BasicUrl + 'ajax/ArticleRelationHandler.ashx?json={"id":"' + id[0] + '","type":"all"}';
        // console.log(catLink);
      
        [detail, catDetail] = await Promise.all([
            httpClien.get(detailLink),
            httpClien.get(catLink)
        ]);

        //文章详情
        if (!detail) {
            return true;
        }

        let data = {};
        data['d_id'] = id[0];
        let k = await cheerio.load(detail['data']);
        data['title'] = k("#main_lblArticleTitle").text();  //标题
        data['abstract'] = k("#allAbstrack").text().replace(/摘　　要：/g, '').replace(/>>收起/g, '');  //摘要
        data['hdclassname'] = k("#hdclassname").val();  //分类
        // console.log(title,abstract,hdclassname)
        k(".article_detail p").each((key, i) => {
            let detailItem = k(i).text().match(/【(.+)】(.+)/);

            if (!detailItem[1] || !detailItem[2]) {  //小标题
                return true;
            }

            sTitle = detailItem[1].replace(/\s+/g, "");
            sText = detailItem[2].replace(/^\s+|\s+$/g, "");

            switch (sTitle) {
                case "作者":
                    data['author'] = sText.replace(/\s+/g, ",");
                    break;
                case "作者单位":
                    data['institution'] = sText;
                    break;
                case "期刊":
                    let journalData = sText.match(/《(.+)》\s+(.+)/)
                    data['journal'] = journalData[1] || '';
                    data['date'] = (journalData[2] || '').split(',')[0] || '';
                    break;
                case "关键词":
                    data['key_word'] = sText.replace(/\s+/g, ",");
                    break;

                case "基金项目":
                    data['fund'] = sText;
                    break;
            }

        });
        //获取typename
        if (!catDetail || !catDetail['data']) {
            console.log(`err:${id[0]}获取typename失败`);
            return true;
        }

        // console.log(catDetail['data']);
        let string = catDetail['data'];
        let results;
        let html = JSON.parse(string['ResultHtml'])[1];
        if(html['totalCount'] == 0) {
            return true;
        } else if(html['totalCount'] <= 12) {
            results = html['results'];
        } else {
            let typeName  = html['typeName'];
            //获取全部的参考文献
            catDetailAll = await httpClien.get(`${BasicUrl}ajax/ArticleRelationHandler.ashx?json={"typeName":"${typeName}","id":"${id[0]}","pageIndex":1,"type":"part"}`);
            if(!catDetailAll || !catDetailAll['data']) {
                console.log(`err:${id[0]}获取参考文献失败`);
                return true;
            }

            results = JSON.parse(catDetailAll['data']['ResultHtml'])['results']; 
        }

        //如果找到目录
        //匹配标号 <span>[1]</span>徐娟[1].完工百分比法在施工企业收入确认中的运用[J].北方经贸,2015,0(7):241-242.
        results.forEach(element => {
            let insertData = data;
            let book;   //附录书籍
            let subMatchs;   //二级匹配
            let books;
            let match = element.match(/\<span\>\[(\d+)\]\<\/span\>(.+)/); //匹配

            if(!match[2]) {
                books = match[1];
            } else {
                books = match[2];
            }

            //判断中英文，确定分词
            let splitWord;
            let baseIndex;
            if(books.search(/[\u4e00-\u9fa5]/)>=0) {  //为中文
                splitWord = ".";
                baseIndex = 1;
            } else {
                splitWord = ",";  
                baseIndex = 2;
            }

            subMatchs = books.split(splitWord);
            
ctx && ctx.websocket.send(`匹配到参考文献：${books}`);
console.log(books);
            //查找书名
            let i;
            for (i = 0; i < subMatchs.length; i++) {
                let ele = subMatchs[i];
                let flag = ele.search(/\[[D|J|N|M|C|R|S|P|Z|DB|CP|EB|OL|CD|MT|DK|A]\]/gi);
                if (flag >= 0) {
                    book = ele.substring(0, flag);
                    // book = ele;
                    break;
                }
            }
            let startIndex = 0;
            if (!book) {   //无法通过书本分类号查找到书本
                subMatchsElse1 = books.match(/(.*)[:|\.|,][《?|'?|"?](.*)[》?|'?|"?][\;|,|\.](.*)/);  //通过暴力方式获取书本
                subMatchsElse2 = books.match(/(.*?)\d{4}[A-Za-z]?[\,|\:|\.](.*?)[\,|\.](.*)/);  //通过暴力方式获取书本
                if(subMatchsElse1 && subMatchsElse1[2]) {
                    i = 2;
                    subMatchs = subMatchsElse1;
                    startIndex = 1;
                }else if(subMatchsElse2 && subMatchsElse2[2]) {
                    i = 2;
                    subMatchs = subMatchsElse2;
                    startIndex = 1;
                } else {
                    i = baseIndex;
                }
                book = subMatchs[i];
            }

            if(book==null) {
                // console.log(`err:${element},count:${html['totalCount']},id:${id[0]}`);
                return true;
            }

            //出版社和印刷年份
            let k = i+1;
            let publish='';
            let publishTime=0;                    
            if(subMatchs[k]) {
                let publishs = subMatchs[k].split(",");
                publish = publishs[0] || '';
                if(isNaN(publishs[1]) == false ) {
                    publishTime = publishs[1] || 0;
                }
            }

            //作者
            let slice = subMatchs.slice(startIndex, i).sort();
            let slices = [];
            slice.forEach(item => {  //去除作者标号
                slices.push(item.replace(/\[\d{1}\]/g, ''));


            });
            let c_author = slices.join(",");
            let timestamp = Date.parse(new Date())/1000;
            insertData = Object.assign({
                c_author,
                c_title:book,
                c_journal:publish,
                c_date:publishTime,
                created_at:timestamp ,
                updated_at:timestamp
            },insertData);
            nssd.insertData(insertData);
        });
        
        
    });
}

function sleep(time) {
    return new Promise((resolve) => setTimeout(resolve, time));
}


exports.websocket = async function (ctx, next) {
    ctx.websocket.on("message", async function(message){  
        let arr = message.split("&");
        let obj = {};
        arr.forEach(ele => {
            let agrs =  ele.split("=")
            obj[agrs[0]] = agrs[1];
        })
        const {pageS = 1 ,pageE = 0, keyword = "马克思主义"} = obj;
        let p = pageS
        for (; p <= pageE; p++) { //页码
            let url = `${BasicUrl}articles/articlesearch.aspx?invokemethod=search&q={"search":"${keyword}","sType":"all","page":"${p}"}&&hidpage=0&&hfldSelectedIds=&`;  //关键词
            ctx.websocket.send("列表URL：" + url);
            console.log(url);
            try {
                 await _do(encodeURI(url), ctx);
            } catch (err) {
                ctx.websocket.send(err);
                console.log(err);
            }
        }
        ctx.websocket.send("end");
    });
}

exports.nssd = nssdCrawler;
